package cn.hy.spider.x6o;

import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONException;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import cn.hy.spider.HeaderUtil;
import cn.hy.spider.HttpClientUtil;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;

import java.io.File;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;

public class AppTitleGet {
    public static void main(String[] args) throws Exception {

        String urlTemplate = "https://x6o.com/api/topics/14/articles?page={}&per_page=80&order=-create_time&include=user%2Ctopics%2Cis_following";
        CloseableHttpClient httpClient = HttpClientUtil.getHttpClient();
        RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(14999).setConnectTimeout(15000)
                .setConnectionRequestTimeout(14999).setProxy(new HttpHost("127.0.0.1", 7890)).build();
        Random random = new Random();
        for (int i = 100; i <= 120; i++) {
            String url = StrUtil.format(urlTemplate, i);
            HttpGet get = new HttpGet(url);
            get.setConfig(requestConfig);
            get.setHeader("User-Agent", HeaderUtil.headers[random.nextInt(HeaderUtil.headers.length)]);
            CloseableHttpResponse response = httpClient.execute(get);
            HttpEntity entity = response.getEntity();
            String entityStr = EntityUtils.toString(entity);
            JSONObject jo = null;
            try {
                jo = JSONUtil.parseObj(entityStr);
            } catch (JSONException e) {
                // ignore
                e.printStackTrace();
                continue;
            }
            JSONArray array = jo.getJSONArray("data");
            File file = FileUtil.newFile("/Users/code/L1/pic-handler/src/main/resources/title.txt");
            Set<String> titleSet = new HashSet<>();
            for (Object obj : array) {
                JSONObject d = (JSONObject) obj;
                String title = d.getStr("title");
                if (StrUtil.isNotBlank(title) && title.contains("JVID")) {
                    continue;
                }
                titleSet.add(title);
            }
            System.out.println("titleSet = " + titleSet.size());
            FileUtil.appendUtf8Lines(titleSet, file);
        }
    }
}
